

<!DOCTYPE html>
<html lang="zh-CN" data-default-color-scheme=auto>



<head>
  <meta charset="UTF-8">
  <link rel="apple-touch-icon" sizes="76x76" href="/img/fluid.png">
  <link rel="icon" href="/img/fluid.png">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, shrink-to-fit=no">
  <meta http-equiv="x-ua-compatible" content="ie=edge">
  
  <meta name="theme-color" content="#2f4154">
  <meta name="author" content="John Doe">
  <meta name="keywords" content="">
  
    <meta name="description" content="“ 常在河边走，哪能不湿鞋。” ——若发现文章内容有误，敬请指正，望不吝赐教，感谢!@[toc]  一、参考资料 视频资料 二、运行环境  windows 10 JDK 8 Hadoop 3.1.3 windows版 IDEA  三、Partition分区 3.1 默认Partitioner分区相关的部分源码 org.apache.hadoop.mapreduce.lib.partition.H">
<meta property="og:type" content="article">
<meta property="og:title" content="Hadoop _ MapReduce学习笔记 _ Partitioner分区 自定义分区策略案例 _ WritableComparable 全排序 _ Combiner、OutputFormat">
<meta property="og:url" content="http://example.com/2022/02/13/Hadoop%20_%20MapReduce%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0%20_%20Partitioner%E5%88%86%E5%8C%BA%20%E8%87%AA%E5%AE%9A%E4%B9%89%E5%88%86%E5%8C%BA%E7%AD%96%E7%95%A5%E6%A1%88%E4%BE%8B%20_%20WritableComparable%20%E5%85%A8%E6%8E%92%E5%BA%8F%20_%20Combiner%E3%80%81OutputFormat/index.html">
<meta property="og:site_name" content="Hexo">
<meta property="og:description" content="“ 常在河边走，哪能不湿鞋。” ——若发现文章内容有误，敬请指正，望不吝赐教，感谢!@[toc]  一、参考资料 视频资料 二、运行环境  windows 10 JDK 8 Hadoop 3.1.3 windows版 IDEA  三、Partition分区 3.1 默认Partitioner分区相关的部分源码 org.apache.hadoop.mapreduce.lib.partition.H">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://img-blog.csdnimg.cn/a2f58014e09546af82005f38974affc5.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/b691238f65f74bbeb7e7fb730973b45d.png">
<meta property="article:published_time" content="2022-02-13T14:15:38.000Z">
<meta property="article:modified_time" content="2022-08-22T15:44:22.953Z">
<meta property="article:author" content="John Doe">
<meta property="article:tag" content="MapReduce">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="https://img-blog.csdnimg.cn/a2f58014e09546af82005f38974affc5.png">
  
  
  
  <title>Hadoop _ MapReduce学习笔记 _ Partitioner分区 自定义分区策略案例 _ WritableComparable 全排序 _ Combiner、OutputFormat - Hexo</title>

  <link  rel="stylesheet" href="https://lib.baomitu.com/twitter-bootstrap/4.6.1/css/bootstrap.min.css" />



  <link  rel="stylesheet" href="https://lib.baomitu.com/github-markdown-css/4.0.0/github-markdown.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/hint.css/2.7.0/hint.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.css" />



<!-- 主题依赖的图标库，不要自行修改 -->
<!-- Do not modify the link that theme dependent icons -->

<link rel="stylesheet" href="//at.alicdn.com/t/font_1749284_hj8rtnfg7um.css">



<link rel="stylesheet" href="//at.alicdn.com/t/font_1736178_lbnruvf0jn.css">


<link  rel="stylesheet" href="/css/main.css" />


  <link id="highlight-css" rel="stylesheet" href="/css/highlight.css" />
  
    <link id="highlight-css-dark" rel="stylesheet" href="/css/highlight-dark.css" />
  




  <script id="fluid-configs">
    var Fluid = window.Fluid || {};
    Fluid.ctx = Object.assign({}, Fluid.ctx)
    var CONFIG = {"hostname":"example.com","root":"/","version":"1.9.2","typing":{"enable":true,"typeSpeed":70,"cursorChar":"_","loop":false,"scope":[]},"anchorjs":{"enable":true,"element":"h1,h2,h3,h4,h5,h6","placement":"left","visible":"hover","icon":""},"progressbar":{"enable":true,"height_px":3,"color":"#29d","options":{"showSpinner":false,"trickleSpeed":100}},"code_language":{"enable":true,"default":"TEXT"},"copy_btn":true,"image_caption":{"enable":true},"image_zoom":{"enable":true,"img_url_replace":["",""]},"toc":{"enable":true,"placement":"right","headingSelector":"h1,h2,h3,h4,h5,h6","collapseDepth":0},"lazyload":{"enable":true,"loading_img":"/img/loading.gif","onlypost":false,"offset_factor":2},"web_analytics":{"enable":false,"follow_dnt":true,"baidu":null,"google":null,"gtag":null,"tencent":{"sid":null,"cid":null},"woyaola":null,"cnzz":null,"leancloud":{"app_id":null,"app_key":null,"server_url":null,"path":"window.location.pathname","ignore_local":false}},"search_path":"/local-search.xml"};

    if (CONFIG.web_analytics.follow_dnt) {
      var dntVal = navigator.doNotTrack || window.doNotTrack || navigator.msDoNotTrack;
      Fluid.ctx.dnt = dntVal && (dntVal.startsWith('1') || dntVal.startsWith('yes') || dntVal.startsWith('on'));
    }
  </script>
  <script  src="/js/utils.js" ></script>
  <script  src="/js/color-schema.js" ></script>
  


  
<meta name="generator" content="Hexo 6.2.0"></head>


<body>
  

  <header>
    

<div class="header-inner" style="height: 70vh;">
  <nav id="navbar" class="navbar fixed-top  navbar-expand-lg navbar-dark scrolling-navbar">
  <div class="container">
    <a class="navbar-brand" href="/">
      <strong>尤 Ni&#39;s Blog</strong>
    </a>

    <button id="navbar-toggler-btn" class="navbar-toggler" type="button" data-toggle="collapse"
            data-target="#navbarSupportedContent"
            aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
      <div class="animated-icon"><span></span><span></span><span></span></div>
    </button>

    <!-- Collapsible content -->
    <div class="collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav ml-auto text-center">
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/">
                <i class="iconfont icon-home-fill"></i>
                首页
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/archives/">
                <i class="iconfont icon-archive-fill"></i>
                归档
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/categories/">
                <i class="iconfont icon-category-fill"></i>
                分类
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/tags/">
                <i class="iconfont icon-tags-fill"></i>
                标签
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/about/">
                <i class="iconfont icon-user-fill"></i>
                关于
              </a>
            </li>
          
        
        
          <li class="nav-item" id="search-btn">
            <a class="nav-link" target="_self" href="javascript:;" data-toggle="modal" data-target="#modalSearch" aria-label="Search">
              &nbsp;<i class="iconfont icon-search"></i>&nbsp;
            </a>
          </li>
          
        
        
          <li class="nav-item" id="color-toggle-btn">
            <a class="nav-link" target="_self" href="javascript:;" aria-label="Color Toggle">&nbsp;<i
                class="iconfont icon-dark" id="color-toggle-icon"></i>&nbsp;</a>
          </li>
        
      </ul>
    </div>
  </div>
</nav>

  

<div id="banner" class="banner" parallax=true
     style="background: url('/img/default.png') no-repeat center center; background-size: cover;">
  <div class="full-bg-img">
    <div class="mask flex-center" style="background-color: rgba(0, 0, 0, 0.3)">
      <div class="banner-text text-center fade-in-up">
        <div class="h2">
          
            <span id="subtitle" data-typed-text="Hadoop _ MapReduce学习笔记 _ Partitioner分区 自定义分区策略案例 _ WritableComparable 全排序 _ Combiner、OutputFormat"></span>
          
        </div>

        
          
  <div class="mt-3">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-author" aria-hidden="true"></i>
        John Doe
      </span>
    
    
      <span class="post-meta">
        <i class="iconfont icon-date-fill" aria-hidden="true"></i>
        <time datetime="2022-02-13 22:15" pubdate>
          2022年2月13日 晚上
        </time>
      </span>
    
  </div>

  <div class="mt-1">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-chart"></i>
        
          23k 字
        
      </span>
    

    
      <span class="post-meta mr-2">
        <i class="iconfont icon-clock-fill"></i>
        
        
        
          193 分钟
        
      </span>
    

    
    
  </div>


        
      </div>

      
    </div>
  </div>
</div>

</div>

  </header>

  <main>
    
      

<div class="container-fluid nopadding-x">
  <div class="row nomargin-x">
    <div class="side-col d-none d-lg-block col-lg-2">
      

    </div>

    <div class="col-lg-8 nopadding-x-md">
      <div class="container nopadding-x-md" id="board-ctn">
        <div id="board">
          <article class="post-content mx-auto">
            <!-- SEO header -->
            <h1 style="display: none">Hadoop _ MapReduce学习笔记 _ Partitioner分区 自定义分区策略案例 _ WritableComparable 全排序 _ Combiner、OutputFormat</h1>
            
              <p class="note note-info">
                
                  
                    本文最后更新于：1 小时前
                  
                
              </p>
            
            
              <div class="markdown-body">
                
                <blockquote>
<p>“ 常在河边走，哪能不湿鞋。” ——若发现文章内容有误，敬请指正，望不吝赐教，感谢!<br>@[toc]</p>
</blockquote>
<h1 id="一、参考资料"><a href="#一、参考资料" class="headerlink" title="一、参考资料"></a>一、参考资料</h1><hr>
<p><a target="_blank" rel="noopener" href="https://www.bilibili.com/video/BV1Qp4y1n7EN?p=95&spm_id_from=pageDriver">视频资料</a></p>
<h1 id="二、运行环境"><a href="#二、运行环境" class="headerlink" title="二、运行环境"></a>二、运行环境</h1><hr>
<ul>
<li>windows 10</li>
<li>JDK 8</li>
<li>Hadoop 3.1.3 windows版</li>
<li>IDEA</li>
</ul>
<h1 id="三、Partition分区"><a href="#三、Partition分区" class="headerlink" title="三、Partition分区"></a>三、Partition分区</h1><hr>
<h2 id="3-1-默认Partitioner分区"><a href="#3-1-默认Partitioner分区" class="headerlink" title="3.1 默认Partitioner分区"></a>3.1 默认Partitioner分区</h2><p>相关的部分源码 <code>org.apache.hadoop.mapreduce.lib.partition.HashPartitioner.java</code>：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">public</span> <span class="hljs-type">int</span> <span class="hljs-title function_">getPartition</span><span class="hljs-params">(K key, V value,</span><br><span class="hljs-params">                        <span class="hljs-type">int</span> numReduceTasks)</span> &#123;<br>  <span class="hljs-keyword">return</span> (key.hashCode() &amp; Integer.MAX_VALUE) % numReduceTasks;<br>&#125;<br></code></pre></td></tr></table></figure>
<p>默认分区是根据Map任务key的hasCode对ReduceTasks个数取模得到的，用户无法控制哪个key存储到哪个分区</p>
<h2 id="3-2-自定义-Parititioner-分区-步骤-amp-案例"><a href="#3-2-自定义-Parititioner-分区-步骤-amp-案例" class="headerlink" title="3.2 自定义 Parititioner 分区 步骤&amp;案例"></a>3.2 自定义 Parititioner 分区 步骤&amp;案例</h2><p><strong>案例：</strong> 按电话号码进行统计，统计手机耗费的总流量，将统计结果按照手机归属地不同省份输出到不同文件中（分区），将手机号135、137,138开头的都放到各自独立的三个文件中，其他开头的放到同一个文件中<br>测试数据：</p>
<figure class="highlight txt"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><code class="hljs txt">1	13736230513	192.196.100.1	www.atguigu.com	2481	24681	200<br>2	13846544121	192.196.100.2			264	0	200<br>3 	13956435636	192.196.100.3			132	1512	200<br>4 	13966251146	192.168.100.1			240	0	404<br>5 	18271575951	192.168.100.2	www.atguigu.com	1527	2106	200<br>6 	84188413	192.168.100.3	www.atguigu.com	4116	1432	200<br>7 	13590439668	192.168.100.4			1116	954	200<br>8 	15910133277	192.168.100.5	www.hao123.com	3156	2936	200<br>9 	13729199489	192.168.100.6			240	0	200<br>10 	13630577991	192.168.100.7	www.shouhu.com	6960	690	200<br>11 	15043685818	192.168.100.8	www.baidu.com	3659	3538	200<br>12 	15959002129	192.168.100.9	www.atguigu.com	1938	180	500<br>13 	13560439638	192.168.100.10			918	4938	200<br>14 	13470253144	192.168.100.11			180	180	200<br>15 	13682846555	192.168.100.12	www.qq.com	1938	2910	200<br>16 	13992314666	192.168.100.13	www.gaga.com	3008	3720	200<br>17 	13509468723	192.168.100.14	www.qinghua.com	7335	110349	404<br>18 	18390173782	192.168.100.15	www.sogou.com	9531	2412	200<br>19 	13975057813	192.168.100.16	www.baidu.com	11058	48243	200<br>20 	13768778790	192.168.100.17			120	120	200<br>21 	13568436656	192.168.100.18	www.alibaba.com	2481	24681	200<br>22 	13568436656	192.168.100.19			1116	954	200<br></code></pre></td></tr></table></figure>
<p><strong>输入数据格式：</strong></p>
<figure class="highlight apache"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs apache"><span class="hljs-attribute">id</span>	手机号码		网络ip			上行流量  下行流量     网络状态码<br><span class="hljs-attribute">7</span> 	<span class="hljs-number">13560436666</span>	<span class="hljs-number">120.196.100.99</span>		<span class="hljs-number">1116</span>		 <span class="hljs-number">954</span>			<span class="hljs-number">200</span><br></code></pre></td></tr></table></figure>
<p><strong>输出数据格式：</strong></p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs yaml"><span class="hljs-number">13560436666</span> 		<span class="hljs-number">1116		      </span><span class="hljs-number">954</span> 			<span class="hljs-number">2070</span><br><span class="hljs-string">手机号码</span>		    <span class="hljs-string">上行流量</span>        <span class="hljs-string">下行流量</span>		<span class="hljs-string">总流量</span><br></code></pre></td></tr></table></figure>
<p><strong>步骤分析：</strong> 读取输入数据、增加ProvincePartitioner分区（四个分区，分别存储135、136,137、其他手机号开头的统计结果）、期望数据输出、Driver驱动类指定自定义的数据分区与ReduceTask个数</p>
<h3 id="3-2-1-编写序列化实体类"><a href="#3-2-1-编写序列化实体类" class="headerlink" title="3.2.1 编写序列化实体类"></a>3.2.1 编写序列化实体类</h3><p>用于map、reduce过程的数据传递<br><code>FlowBean.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.partitioner2;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Writable;<br><br><span class="hljs-keyword">import</span> java.io.DataInput;<br><span class="hljs-keyword">import</span> java.io.DataOutput;<br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-comment">/**</span><br><span class="hljs-comment"> * 1. 定义类实现 writable 接口</span><br><span class="hljs-comment"> * 2. 重写序列化和反序列化方法</span><br><span class="hljs-comment"> * 3. 重写无参构造</span><br><span class="hljs-comment"> * 4. toString方法</span><br><span class="hljs-comment"> *</span><br><span class="hljs-comment"> */</span><br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">FlowBean</span> <span class="hljs-keyword">implements</span> <span class="hljs-title class_">Writable</span> &#123;<br>    <span class="hljs-keyword">private</span> <span class="hljs-type">long</span> upFlow;    <span class="hljs-comment">// 上行流量</span><br>    <span class="hljs-keyword">private</span> <span class="hljs-type">long</span> downFlow;  <span class="hljs-comment">// 下行流量</span><br>    <span class="hljs-keyword">private</span> <span class="hljs-type">long</span> sumFlow;   <span class="hljs-comment">// 总流量</span><br><br>    <span class="hljs-comment">// 无参构造</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-title function_">FlowBean</span><span class="hljs-params">()</span>&#123;&#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-type">long</span> <span class="hljs-title function_">getUpFlow</span><span class="hljs-params">()</span> &#123;<br>        <span class="hljs-keyword">return</span> upFlow;<br>    &#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">setUpFlow</span><span class="hljs-params">(<span class="hljs-type">long</span> upFlow)</span> &#123;<br>        <span class="hljs-built_in">this</span>.upFlow = upFlow;<br>    &#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-type">long</span> <span class="hljs-title function_">getDownFlow</span><span class="hljs-params">()</span> &#123;<br>        <span class="hljs-keyword">return</span> downFlow;<br>    &#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">setDownFlow</span><span class="hljs-params">(<span class="hljs-type">long</span> downFlow)</span> &#123;<br>        <span class="hljs-built_in">this</span>.downFlow = downFlow;<br>    &#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-type">long</span> <span class="hljs-title function_">getSumFlow</span><span class="hljs-params">()</span> &#123;<br>        <span class="hljs-keyword">return</span> sumFlow;<br>    &#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">setSumFlow</span><span class="hljs-params">()</span> &#123;<br>        <span class="hljs-built_in">this</span>.sumFlow = <span class="hljs-built_in">this</span>.upFlow + <span class="hljs-built_in">this</span>.downFlow;<br>    &#125;<br><br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">write</span><span class="hljs-params">(DataOutput out)</span> <span class="hljs-keyword">throws</span> IOException &#123;<br>        out.writeLong(upFlow);<br>        out.writeLong(downFlow);<br>        out.writeLong(sumFlow);<br>    &#125;<br><br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">readFields</span><span class="hljs-params">(DataInput in)</span> <span class="hljs-keyword">throws</span> IOException &#123;<br>        <span class="hljs-built_in">this</span>.upFlow = in.readLong();<br>        <span class="hljs-built_in">this</span>.downFlow = in.readLong();<br>        <span class="hljs-built_in">this</span>.sumFlow = in.readLong();<br>    &#125;<br><br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> String <span class="hljs-title function_">toString</span><span class="hljs-params">()</span> &#123;<br>        <span class="hljs-keyword">return</span> upFlow + <span class="hljs-string">&quot;\t&quot;</span> + downFlow + <span class="hljs-string">&quot;\t&quot;</span> + sumFlow;<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>
<h3 id="3-2-2-自定义类继承Partitioner"><a href="#3-2-2-自定义类继承Partitioner" class="headerlink" title="3.2.2 自定义类继承Partitioner"></a>3.2.2 自定义类继承Partitioner</h3><p>需要在继承类里重写 <strong>getPartition()</strong> 方法<br><code>ProvincePartitioner.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.partitioner2;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Partitioner;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">ProvincePartitioner</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">Partitioner</span>&lt;Text, FlowBean&gt; &#123;<br><br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-type">int</span> <span class="hljs-title function_">getPartition</span><span class="hljs-params">(Text text, FlowBean flowBean, <span class="hljs-type">int</span> partition)</span> &#123;<br>        <span class="hljs-comment">// text 手机号</span><br>        <span class="hljs-type">String</span> <span class="hljs-variable">phone</span> <span class="hljs-operator">=</span> text.toString();<br>        <span class="hljs-type">String</span> <span class="hljs-variable">prePhone</span> <span class="hljs-operator">=</span> phone.substring(<span class="hljs-number">0</span>, <span class="hljs-number">3</span>);<br>        <span class="hljs-keyword">if</span>(<span class="hljs-string">&quot;135&quot;</span>.equals(prePhone))&#123;<br>            partition = <span class="hljs-number">0</span>;<br>        &#125; <span class="hljs-keyword">else</span> <span class="hljs-keyword">if</span> (<span class="hljs-string">&quot;136&quot;</span>.equals(prePhone))&#123;<br>            partition = <span class="hljs-number">1</span>;<br>        &#125; <span class="hljs-keyword">else</span> <span class="hljs-keyword">if</span> (<span class="hljs-string">&quot;137&quot;</span>.equals(prePhone))&#123;<br>            partition = <span class="hljs-number">2</span>;<br>        &#125; <span class="hljs-keyword">else</span><br>            partition = <span class="hljs-number">3</span>;<br>        <span class="hljs-keyword">return</span> partition;<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>

<h3 id="3-2-3-编写自定义的Map、Reduce类"><a href="#3-2-3-编写自定义的Map、Reduce类" class="headerlink" title="3.2.3 编写自定义的Map、Reduce类"></a>3.2.3 编写自定义的Map、Reduce类</h3><p><code>FlowMapper.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.partitioner2;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.LongWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Mapper;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">FlowMapper</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">Mapper</span>&lt;LongWritable, Text, Text, FlowBean&gt; &#123;<br>    <span class="hljs-keyword">private</span> <span class="hljs-type">Text</span> <span class="hljs-variable">outputKey</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Text</span>();<br>    <span class="hljs-keyword">private</span> <span class="hljs-type">FlowBean</span> <span class="hljs-variable">outputValue</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">FlowBean</span>();<br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">protected</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">map</span><span class="hljs-params">(LongWritable key, Text value, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>        <span class="hljs-comment">// 1.获取一行  7 	13560436666	120.196.100.99		1116		 954			200</span><br>        <span class="hljs-type">String</span> <span class="hljs-variable">line</span> <span class="hljs-operator">=</span> value.toString();<br>        <span class="hljs-comment">// 2.切割成 [7,13560436666,120.196.100.99,1116,954,200]</span><br>        String[] split = line.split(<span class="hljs-string">&quot;\t&quot;</span>);<br>        <span class="hljs-comment">// 3. 获取想要的数据: 手机号 13560436666, 上行流量和下行流量: 1116、954</span><br>        <span class="hljs-type">String</span> <span class="hljs-variable">phone</span> <span class="hljs-operator">=</span> split[<span class="hljs-number">1</span>];<br>        <span class="hljs-comment">// 顺序的话数据有残缺，故逆序取上行、下行流量</span><br>        <span class="hljs-type">String</span> <span class="hljs-variable">up</span> <span class="hljs-operator">=</span> split[split.length - <span class="hljs-number">3</span>];<br>        <span class="hljs-type">String</span> <span class="hljs-variable">down</span> <span class="hljs-operator">=</span> split[split.length - <span class="hljs-number">2</span>];<br><br>        <span class="hljs-comment">// 4. 封装</span><br>        outputKey.set(phone);<br>        outputValue.setUpFlow(Long.parseLong(up));<br>        outputValue.setDownFlow(Long.parseLong(down));<br>        outputValue.setSumFlow();<br>        <span class="hljs-comment">// 5. 写出</span><br>        context.write(outputKey, outputValue);<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>

<p><code>FlowReducer.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.partitioner2;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Reducer;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">FlowReducer</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">Reducer</span>&lt;Text, FlowBean, Text, FlowBean&gt; &#123;<br>    <span class="hljs-keyword">private</span> <span class="hljs-type">FlowBean</span> <span class="hljs-variable">outputValue</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">FlowBean</span>();<br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">protected</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">reduce</span><span class="hljs-params">(Text key, Iterable&lt;FlowBean&gt; values, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>        <span class="hljs-comment">// 1. 遍历集合，累加上下行流量</span><br>        <span class="hljs-type">long</span> <span class="hljs-variable">totalUp</span> <span class="hljs-operator">=</span> <span class="hljs-number">0</span>;<br>        <span class="hljs-type">long</span> <span class="hljs-variable">totalDown</span> <span class="hljs-operator">=</span> <span class="hljs-number">0</span>;<br>        <span class="hljs-keyword">for</span> (FlowBean value : values) &#123;<br>            totalUp += value.getUpFlow();<br>            totalDown += value.getDownFlow();<br>        &#125;<br>        <span class="hljs-comment">// 2. 封装 outputKey，outputValue</span><br>        outputValue.setUpFlow(totalUp);<br>        outputValue.setDownFlow(totalDown);<br>        outputValue.setSumFlow();<br><br>        <span class="hljs-comment">// 3. 写出</span><br>        context.write(key, outputValue);<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>
<h3 id="3-2-4-Driver类-指定Partitioner"><a href="#3-2-4-Driver类-指定Partitioner" class="headerlink" title="3.2.4 Driver类-指定Partitioner"></a>3.2.4 Driver类-指定Partitioner</h3><p><code>FlowDriver.java</code> Driver驱动类</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.partitioner2;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.conf.Configuration;<br><span class="hljs-keyword">import</span> org.apache.hadoop.fs.Path;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Job;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.lib.input.FileInputFormat;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">FlowDriver</span> &#123;<br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">static</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">main</span><span class="hljs-params">(String[] args)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException, ClassNotFoundException &#123;<br>        <span class="hljs-comment">// 1. 创建连接</span><br>        <span class="hljs-type">Configuration</span> <span class="hljs-variable">conf</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Configuration</span>();<br>        <span class="hljs-type">Job</span> <span class="hljs-variable">job</span> <span class="hljs-operator">=</span> Job.getInstance(conf);<br>        <span class="hljs-comment">// 2. 设置jar</span><br>        job.setJarByClass(FlowDriver.class);<br>        <span class="hljs-comment">// 3. 关联 Mapper 、Reducer</span><br>        job.setMapperClass(FlowMapper.class);<br>        job.setReducerClass(FlowReducer.class);<br>        <span class="hljs-comment">// 4. 设置 mapper的输出值 kv类型</span><br>        job.setMapOutputKeyClass(Text.class);<br>        job.setMapOutputValueClass(FlowBean.class);<br>        <span class="hljs-comment">// 5. 设置 最终的输出值 kv类型</span><br>        job.setOutputKeyClass(Text.class);<br>        job.setOutputValueClass(FlowBean.class);<br><br>        <span class="hljs-comment">// 设置分区自定义类（分区策略）</span><br>        job.setPartitionerClass(ProvincePartitioner.class);<br>        <span class="hljs-comment">// 设置分区数</span><br>        job.setNumReduceTasks(<span class="hljs-number">4</span>);<br><br>        <span class="hljs-comment">// 6. 设置数据的输入路径和输出路径</span><br>        FileInputFormat.setInputPaths(job, <span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;input&quot;</span>));<br>        FileOutputFormat.setOutputPath(job, <span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;output3&quot;</span>));<br>        <span class="hljs-comment">// 7. 提交Job</span><br>        <span class="hljs-type">boolean</span> <span class="hljs-variable">result</span> <span class="hljs-operator">=</span> job.waitForCompletion(<span class="hljs-literal">true</span>);<br>        System.exit(result ? <span class="hljs-number">0</span> : <span class="hljs-number">1</span>);<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>

<p>其中分区相关的操作为：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-comment">// 设置分区自定义类（分区策略）</span><br>job.setPartitionerClass(ProvincePartitioner.class);<br><span class="hljs-comment">// 设置分区数</span><br>job.setNumReduceTasks(<span class="hljs-number">4</span>);<br></code></pre></td></tr></table></figure>

<h3 id="3-2-5-运行结果"><a href="#3-2-5-运行结果" class="headerlink" title="3.2.5 运行结果"></a>3.2.5 运行结果</h3><p>分区1的文件内容：<code>part-r-00000</code></p>
<figure class="highlight apache"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><code class="hljs apache"><span class="hljs-attribute">13509468723</span>	<span class="hljs-number">7335</span>	<span class="hljs-number">110349</span>	<span class="hljs-number">117684</span><br><span class="hljs-attribute">13560439638</span>	<span class="hljs-number">918</span>	<span class="hljs-number">4938</span>	<span class="hljs-number">5856</span><br><span class="hljs-attribute">13568436656</span>	<span class="hljs-number">3597</span>	<span class="hljs-number">25635</span>	<span class="hljs-number">29232</span><br><span class="hljs-attribute">13590439668</span>	<span class="hljs-number">1116</span>	<span class="hljs-number">954</span>	<span class="hljs-number">2070</span><br></code></pre></td></tr></table></figure>
<p>分区2的文件内容：<code>part-r-00001</code></p>
<figure class="highlight apache"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs apache"><span class="hljs-attribute">13630577991</span>	<span class="hljs-number">6960</span>	<span class="hljs-number">690</span>	<span class="hljs-number">7650</span><br><span class="hljs-attribute">13682846555</span>	<span class="hljs-number">1938</span>	<span class="hljs-number">2910</span>	<span class="hljs-number">4848</span><br></code></pre></td></tr></table></figure>
<p>分区3的文件内容：<code>part-r-00002</code></p>
<figure class="highlight apache"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><code class="hljs apache"><span class="hljs-attribute">13729199489</span>	<span class="hljs-number">240</span>	<span class="hljs-number">0</span>	<span class="hljs-number">240</span><br><span class="hljs-attribute">13736230513</span>	<span class="hljs-number">2481</span>	<span class="hljs-number">24681</span>	<span class="hljs-number">27162</span><br><span class="hljs-attribute">13768778790</span>	<span class="hljs-number">120</span>	<span class="hljs-number">120</span>	<span class="hljs-number">240</span><br></code></pre></td></tr></table></figure>

<p>分区4的文件内容：<code>part-r-00002</code></p>
<figure class="highlight apache"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><code class="hljs apache"><span class="hljs-attribute">13470253144</span>	<span class="hljs-number">180</span>	<span class="hljs-number">180</span>	<span class="hljs-number">360</span><br><span class="hljs-attribute">13846544121</span>	<span class="hljs-number">264</span>	<span class="hljs-number">0</span>	<span class="hljs-number">264</span><br><span class="hljs-attribute">13956435636</span>	<span class="hljs-number">132</span>	<span class="hljs-number">1512</span>	<span class="hljs-number">1644</span><br><span class="hljs-attribute">13966251146</span>	<span class="hljs-number">240</span>	<span class="hljs-number">0</span>	<span class="hljs-number">240</span><br><span class="hljs-attribute">13975057813</span>	<span class="hljs-number">11058</span>	<span class="hljs-number">48243</span>	<span class="hljs-number">59301</span><br><span class="hljs-attribute">13992314666</span>	<span class="hljs-number">3008</span>	<span class="hljs-number">3720</span>	<span class="hljs-number">6728</span><br><span class="hljs-attribute">15043685818</span>	<span class="hljs-number">3659</span>	<span class="hljs-number">3538</span>	<span class="hljs-number">7197</span><br><span class="hljs-attribute">15910133277</span>	<span class="hljs-number">3156</span>	<span class="hljs-number">2936</span>	<span class="hljs-number">6092</span><br><span class="hljs-attribute">15959002129</span>	<span class="hljs-number">1938</span>	<span class="hljs-number">180</span>	<span class="hljs-number">2118</span><br><span class="hljs-attribute">18271575951</span>	<span class="hljs-number">1527</span>	<span class="hljs-number">2106</span>	<span class="hljs-number">3633</span><br><span class="hljs-attribute">18390173782</span>	<span class="hljs-number">9531</span>	<span class="hljs-number">2412</span>	<span class="hljs-number">11943</span><br><span class="hljs-attribute">84188413</span>	<span class="hljs-number">4116</span>	<span class="hljs-number">1432</span>	<span class="hljs-number">5548</span><br></code></pre></td></tr></table></figure>

<h3 id="3-2-6-案例分析"><a href="#3-2-6-案例分析" class="headerlink" title="3.2.6 案例分析"></a>3.2.6 案例分析</h3><p>假设自定义分区数为4，则：<br>情况一： <code>job.setNumReduceTasks(1)</code> 程序正常运行，只不过只产生一个输出文件<br>情况二：- <code>job.setNumReduceTasks(2)</code> 程序报错<br>情况三：- <code>job.setNumReduceTasks(5)</code> 大于4，程序正常运行，不过会产生空的分区文件 part-r-00005</p>
<h3 id="3-2-7-分区总结"><a href="#3-2-7-分区总结" class="headerlink" title="3.2.7 分区总结"></a>3.2.7 分区总结</h3><p>设 $r$ 表示ReduceTask的数量，而 $g$ 表示getPartition的结果数，则有：</p>
<ul>
<li>当 $r &gt; g$，则会多产生几个空的输出文件part-r-000xx</li>
<li>当 $1&lt;r&lt;g$，则有一部分分区数据无处安放，程序会抛出异常</li>
<li>当 $r  &#x3D; 1$，则不管 MapTask端输出多少个分区文件，最终结果都提交给一个ReduceTask，最终就只会产生一个结果文件 part-r-00000</li>
<li>分区必须从0开始，逐一累加</li>
</ul>
<h1 id="四、MapReduce-排序"><a href="#四、MapReduce-排序" class="headerlink" title="四、MapReduce 排序"></a>四、MapReduce 排序</h1><hr>
<p>排序是MapReduce框架中最重要的操作之一</p>
<p>MapTask 和 ReduceTask 均会对数据按照key进行排序。</p>
<p>该操作属于Hadoop的默认行为，任何应用程序中的数据均会被排序，而不管逻辑上是否需要。</p>
<p>默认排序是按照字典顺序排序，且实现该排序的方法是快速排序。</p>
<p>对于MapTask，它会将处理的结果暂时放到环形缓冲区，当环形缓冲区使用率达到一定阈值后，再对缓冲区中的数据进行一次<strong>快速排序</strong>，并将这些有序数据溢写到磁盘上，而当数据处理完毕后，它会对磁盘上所有文件进行 <strong>归并排序</strong>。</p>
<p>对于ReduceTask，它从每个MapTask上远程拷贝响应的数据文件，如果文件大小超过一定阈值，则溢写到磁盘上，否则存储在内存中。如果磁盘上文件数目达到一定阈值，则进行一次合并后将数据溢写到磁盘上。当所有数据拷贝完毕后，<strong>ReduceTask统一对内存和磁盘上的所有数据进行一次归并排序。</strong></p>
<h2 id="4-1-排序分类"><a href="#4-1-排序分类" class="headerlink" title="4.1 排序分类"></a>4.1 排序分类</h2><h3 id="4-1-1-部分排序"><a href="#4-1-1-部分排序" class="headerlink" title="4.1.1 部分排序"></a>4.1.1 部分排序</h3><p>MapReduce 根据输入记录的键对数据集排序。保证输出的每个文件内部有序。</p>
<h3 id="4-1-2-全排序"><a href="#4-1-2-全排序" class="headerlink" title="4.1.2 全排序"></a>4.1.2 全排序</h3><p>最终输出结果只有一个文件，且文件内部有序。实现方法是只设置一个ReduceTask。<br>但该方法在处理大型文件时效率极低，因为一台机器处理所有文件，完全丧失了MapReduce所提供的并行架构。</p>
<h3 id="4-1-3-辅助排序（GroupingComparator分组）"><a href="#4-1-3-辅助排序（GroupingComparator分组）" class="headerlink" title="4.1.3 辅助排序（GroupingComparator分组）"></a>4.1.3 辅助排序（GroupingComparator分组）</h3><p>在Reduce端对key进行分组。应用于：在接收的key为bean对象时，想让一个或几个字段相同（全部字段比较不相同）的key进入到同一个reduce方法时，可以采用分组排序。</p>
<h3 id="4-1-4-二次排序-x2F-自定义排序"><a href="#4-1-4-二次排序-x2F-自定义排序" class="headerlink" title="4.1.4 二次排序 &#x2F; 自定义排序"></a>4.1.4 二次排序 &#x2F; 自定义排序</h3><p>在自定义排序过程中，如果compareTo中的判断条件为两个即为二次排序</p>
<h2 id="4-2-WritableComparable-排序案例（全排序）"><a href="#4-2-WritableComparable-排序案例（全排序）" class="headerlink" title="4.2 WritableComparable 排序案例（全排序）"></a>4.2 WritableComparable 排序案例（全排序）</h2><p><strong>需求：</strong> 根据提供的测试数据，手机号、上行流量、下行流量，总流量四个指标中的总流量倒序排序，总流量相同的话就根据下行流量进行正序排序</p>
<p> 测试数据：<br> <figure class="highlight apache"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><code class="hljs apache"><span class="hljs-attribute">13470253144</span>	<span class="hljs-number">180</span>	<span class="hljs-number">180</span>	<span class="hljs-number">360</span><br><span class="hljs-attribute">13509468723</span>	<span class="hljs-number">7335</span>	<span class="hljs-number">110349</span>	<span class="hljs-number">117684</span><br><span class="hljs-attribute">13560439638</span>	<span class="hljs-number">918</span>	<span class="hljs-number">4938</span>	<span class="hljs-number">5856</span><br><span class="hljs-attribute">13568436656</span>	<span class="hljs-number">3597</span>	<span class="hljs-number">25635</span>	<span class="hljs-number">29232</span><br><span class="hljs-attribute">13590439668</span>	<span class="hljs-number">1116</span>	<span class="hljs-number">954</span>	<span class="hljs-number">2070</span><br><span class="hljs-attribute">13630577991</span>	<span class="hljs-number">6960</span>	<span class="hljs-number">690</span>	<span class="hljs-number">7650</span><br><span class="hljs-attribute">13682846555</span>	<span class="hljs-number">1938</span>	<span class="hljs-number">2910</span>	<span class="hljs-number">4848</span><br><span class="hljs-attribute">13729199489</span>	<span class="hljs-number">240</span>	<span class="hljs-number">0</span>	<span class="hljs-number">240</span><br><span class="hljs-attribute">13736230513</span>	<span class="hljs-number">2481</span>	<span class="hljs-number">24681</span>	<span class="hljs-number">27162</span><br><span class="hljs-attribute">13768778790</span>	<span class="hljs-number">120</span>	<span class="hljs-number">120</span>	<span class="hljs-number">240</span><br><span class="hljs-attribute">13846544121</span>	<span class="hljs-number">264</span>	<span class="hljs-number">0</span>	<span class="hljs-number">264</span><br><span class="hljs-attribute">13956435636</span>	<span class="hljs-number">132</span>	<span class="hljs-number">1512</span>	<span class="hljs-number">1644</span><br><span class="hljs-attribute">13966251146</span>	<span class="hljs-number">240</span>	<span class="hljs-number">0</span>	<span class="hljs-number">240</span><br><span class="hljs-attribute">13975057813</span>	<span class="hljs-number">11058</span>	<span class="hljs-number">48243</span>	<span class="hljs-number">59301</span><br><span class="hljs-attribute">13992314666</span>	<span class="hljs-number">3008</span>	<span class="hljs-number">3720</span>	<span class="hljs-number">6728</span><br><span class="hljs-attribute">15043685818</span>	<span class="hljs-number">3659</span>	<span class="hljs-number">3538</span>	<span class="hljs-number">7197</span><br><span class="hljs-attribute">15910133277</span>	<span class="hljs-number">3156</span>	<span class="hljs-number">2936</span>	<span class="hljs-number">6092</span><br><span class="hljs-attribute">15959002129</span>	<span class="hljs-number">1938</span>	<span class="hljs-number">180</span>	<span class="hljs-number">2118</span><br><span class="hljs-attribute">18271575951</span>	<span class="hljs-number">1527</span>	<span class="hljs-number">2106</span>	<span class="hljs-number">3633</span><br><span class="hljs-attribute">18390173782</span>	<span class="hljs-number">9531</span>	<span class="hljs-number">2412</span>	<span class="hljs-number">11943</span><br><span class="hljs-attribute">84188413</span>	<span class="hljs-number">4116</span>	<span class="hljs-number">1432</span>	<span class="hljs-number">5548</span><br></code></pre></td></tr></table></figure></p>
<p><strong>1. 序列化对象</strong>，包括以总流量、下行流量大小关系为准的排序规则 <code>FlowBean.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.writableComparable;<br><br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Writable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.WritableComparable;<br><br><span class="hljs-keyword">import</span> java.io.DataInput;<br><span class="hljs-keyword">import</span> java.io.DataOutput;<br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">FlowBean</span> <span class="hljs-keyword">implements</span> <span class="hljs-title class_">WritableComparable</span>&lt;FlowBean&gt; &#123;<br>    <span class="hljs-keyword">private</span> <span class="hljs-type">long</span> upFlow;    <span class="hljs-comment">// 上行流量</span><br>    <span class="hljs-keyword">private</span> <span class="hljs-type">long</span> downFlow;  <span class="hljs-comment">// 下行流量</span><br>    <span class="hljs-keyword">private</span> <span class="hljs-type">long</span> sumFlow;   <span class="hljs-comment">// 总流量</span><br><br>    <span class="hljs-comment">// 无参构造</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-title function_">FlowBean</span><span class="hljs-params">()</span>&#123;&#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-type">long</span> <span class="hljs-title function_">getUpFlow</span><span class="hljs-params">()</span> &#123;<br>        <span class="hljs-keyword">return</span> upFlow;<br>    &#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">setUpFlow</span><span class="hljs-params">(<span class="hljs-type">long</span> upFlow)</span> &#123;<br>        <span class="hljs-built_in">this</span>.upFlow = upFlow;<br>    &#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-type">long</span> <span class="hljs-title function_">getDownFlow</span><span class="hljs-params">()</span> &#123;<br>        <span class="hljs-keyword">return</span> downFlow;<br>    &#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">setDownFlow</span><span class="hljs-params">(<span class="hljs-type">long</span> downFlow)</span> &#123;<br>        <span class="hljs-built_in">this</span>.downFlow = downFlow;<br>    &#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-type">long</span> <span class="hljs-title function_">getSumFlow</span><span class="hljs-params">()</span> &#123;<br>        <span class="hljs-keyword">return</span> sumFlow;<br>    &#125;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">setSumFlow</span><span class="hljs-params">()</span> &#123;<br>        <span class="hljs-built_in">this</span>.sumFlow = <span class="hljs-built_in">this</span>.upFlow + <span class="hljs-built_in">this</span>.downFlow;<br>    &#125;<br><br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">write</span><span class="hljs-params">(DataOutput out)</span> <span class="hljs-keyword">throws</span> IOException &#123;<br>        out.writeLong(upFlow);<br>        out.writeLong(downFlow);<br>        out.writeLong(sumFlow);<br>    &#125;<br><br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">readFields</span><span class="hljs-params">(DataInput in)</span> <span class="hljs-keyword">throws</span> IOException &#123;<br>        <span class="hljs-built_in">this</span>.upFlow = in.readLong();<br>        <span class="hljs-built_in">this</span>.downFlow = in.readLong();<br>        <span class="hljs-built_in">this</span>.sumFlow = in.readLong();<br>    &#125;<br><br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> String <span class="hljs-title function_">toString</span><span class="hljs-params">()</span> &#123;<br>        <span class="hljs-keyword">return</span> upFlow + <span class="hljs-string">&quot;\t&quot;</span> + downFlow + <span class="hljs-string">&quot;\t&quot;</span> + sumFlow;<br>    &#125;<br><br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-type">int</span> <span class="hljs-title function_">compareTo</span><span class="hljs-params">(FlowBean o)</span> &#123;<br>        <span class="hljs-comment">// 总流量的倒序排序</span><br>        <span class="hljs-keyword">if</span>(<span class="hljs-built_in">this</span>.sumFlow &gt; o.sumFlow)&#123;<br>            <span class="hljs-keyword">return</span> -<span class="hljs-number">1</span>;<br>        &#125; <span class="hljs-keyword">else</span> <span class="hljs-keyword">if</span>(<span class="hljs-built_in">this</span>.sumFlow &lt; o.sumFlow) &#123;<br>            <span class="hljs-keyword">return</span> <span class="hljs-number">1</span>;<br>        &#125;<br>        <span class="hljs-keyword">else</span>&#123;<br>            <span class="hljs-comment">// 总流量相同， 下行流量正序排序</span><br>            <span class="hljs-keyword">if</span>(<span class="hljs-built_in">this</span>.downFlow &gt; o.downFlow)<br>                <span class="hljs-keyword">return</span> <span class="hljs-number">1</span>;<br>            <span class="hljs-keyword">else</span> <span class="hljs-keyword">if</span>(<span class="hljs-built_in">this</span>.downFlow &lt; o.downFlow)<br>                <span class="hljs-keyword">return</span> -<span class="hljs-number">1</span>;<br>            <span class="hljs-keyword">else</span> <span class="hljs-keyword">return</span> <span class="hljs-number">0</span>;<br>        &#125;<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>


<p><strong>2. Mapper 类：</strong>  <code>FlowMapper.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.writableComparable;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.LongWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Mapper;<br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">FlowMapper</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">Mapper</span>&lt;LongWritable, Text, FlowBean, Text&gt; &#123;<br>    <span class="hljs-keyword">private</span> <span class="hljs-type">FlowBean</span> <span class="hljs-variable">outputKey</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">FlowBean</span>();<br>    <span class="hljs-keyword">private</span> <span class="hljs-type">Text</span> <span class="hljs-variable">outputValue</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Text</span>();<br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">protected</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">map</span><span class="hljs-params">(LongWritable key, Text value, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>       <span class="hljs-comment">// 获取一行</span><br>        <span class="hljs-type">String</span> <span class="hljs-variable">line</span> <span class="hljs-operator">=</span> value.toString();<br>        <span class="hljs-comment">// 切割</span><br>        String[] split = line.split(<span class="hljs-string">&quot;\t&quot;</span>);<br>        <span class="hljs-comment">// 封装</span><br>        outputValue.set(split[<span class="hljs-number">0</span>]);<br>        outputKey.setUpFlow(Long.parseLong(split[<span class="hljs-number">1</span>]));<br>        outputKey.setDownFlow(Long.parseLong(split[<span class="hljs-number">2</span>]));<br>        outputKey.setSumFlow();<br>        <span class="hljs-comment">// 写出</span><br>        context.write(outputKey, outputValue);<br>    &#125;<br>&#125;<br><br></code></pre></td></tr></table></figure>

<p><strong>3. Reduce类：</strong>  <code>FlowReducer.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.writableComparable;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Reducer;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">FlowReducer</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">Reducer</span>&lt;FlowBean, Text, Text, FlowBean&gt; &#123;<br>    <span class="hljs-keyword">private</span> <span class="hljs-type">Text</span> <span class="hljs-variable">outputValue</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Text</span>();<br><br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">protected</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">reduce</span><span class="hljs-params">(FlowBean key, Iterable&lt;Text&gt; values, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>        <span class="hljs-keyword">for</span> (Text value : values) &#123;<br>            context.write(value, key);<br>        &#125;<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>

<p><strong>4. 驱动类：</strong> <code>FlowDriver.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.writableComparable;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.conf.Configuration;<br><span class="hljs-keyword">import</span> org.apache.hadoop.fs.Path;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Job;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.lib.input.FileInputFormat;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">FlowDriver</span> &#123;<br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">static</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">main</span><span class="hljs-params">(String[] args)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException, ClassNotFoundException &#123;<br>        <span class="hljs-comment">// 1. 创建连接</span><br>        <span class="hljs-type">Configuration</span> <span class="hljs-variable">conf</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Configuration</span>();<br>        <span class="hljs-type">Job</span> <span class="hljs-variable">job</span> <span class="hljs-operator">=</span> Job.getInstance(conf);<br>        <span class="hljs-comment">// 2. 设置jar</span><br>        job.setJarByClass(FlowDriver.class);<br>        <span class="hljs-comment">// 3. 关联 Mapper 、Reducer</span><br>        job.setMapperClass(FlowMapper.class);<br>        job.setReducerClass(FlowReducer.class);<br>        <span class="hljs-comment">// 4. 设置 mapper的输出值 kv类型</span><br>        job.setMapOutputKeyClass(FlowBean.class);<br>        job.setMapOutputValueClass(Text.class);<br>        <span class="hljs-comment">// 5. 设置 最终的输出值 kv类型</span><br>        job.setOutputKeyClass(Text.class);<br>        job.setOutputValueClass(FlowBean.class);<br><br>        <span class="hljs-comment">// 6. 设置数据的输入路径和输出路径</span><br>        FileInputFormat.setInputPaths(job, <span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;input2&quot;</span>));<br>        FileOutputFormat.setOutputPath(job, <span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;output2&quot;</span>));<br>        <span class="hljs-comment">// 7. 提交Job</span><br>        <span class="hljs-type">boolean</span> <span class="hljs-variable">result</span> <span class="hljs-operator">=</span> job.waitForCompletion(<span class="hljs-literal">true</span>);<br>        System.exit(result ? <span class="hljs-number">0</span> : <span class="hljs-number">1</span>);<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>
<p><strong>5. 运行结果：</strong></p>
<figure class="highlight apache"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><code class="hljs apache"><span class="hljs-attribute">13509468723</span>	<span class="hljs-number">7335</span>	<span class="hljs-number">110349</span>	<span class="hljs-number">117684</span><br><span class="hljs-attribute">13975057813</span>	<span class="hljs-number">11058</span>	<span class="hljs-number">48243</span>	<span class="hljs-number">59301</span><br><span class="hljs-attribute">13568436656</span>	<span class="hljs-number">3597</span>	<span class="hljs-number">25635</span>	<span class="hljs-number">29232</span><br><span class="hljs-attribute">13736230513</span>	<span class="hljs-number">2481</span>	<span class="hljs-number">24681</span>	<span class="hljs-number">27162</span><br><span class="hljs-attribute">18390173782</span>	<span class="hljs-number">9531</span>	<span class="hljs-number">2412</span>	<span class="hljs-number">11943</span><br><span class="hljs-attribute">13630577991</span>	<span class="hljs-number">6960</span>	<span class="hljs-number">690</span>	<span class="hljs-number">7650</span><br><span class="hljs-attribute">15043685818</span>	<span class="hljs-number">3659</span>	<span class="hljs-number">3538</span>	<span class="hljs-number">7197</span><br><span class="hljs-attribute">13992314666</span>	<span class="hljs-number">3008</span>	<span class="hljs-number">3720</span>	<span class="hljs-number">6728</span><br><span class="hljs-attribute">15910133277</span>	<span class="hljs-number">3156</span>	<span class="hljs-number">2936</span>	<span class="hljs-number">6092</span><br><span class="hljs-attribute">13560439638</span>	<span class="hljs-number">918</span>	<span class="hljs-number">4938</span>	<span class="hljs-number">5856</span><br><span class="hljs-attribute">84188413</span>	<span class="hljs-number">4116</span>	<span class="hljs-number">1432</span>	<span class="hljs-number">5548</span><br><span class="hljs-attribute">13682846555</span>	<span class="hljs-number">1938</span>	<span class="hljs-number">2910</span>	<span class="hljs-number">4848</span><br><span class="hljs-attribute">18271575951</span>	<span class="hljs-number">1527</span>	<span class="hljs-number">2106</span>	<span class="hljs-number">3633</span><br><span class="hljs-attribute">15959002129</span>	<span class="hljs-number">1938</span>	<span class="hljs-number">180</span>	<span class="hljs-number">2118</span><br><span class="hljs-attribute">13590439668</span>	<span class="hljs-number">1116</span>	<span class="hljs-number">954</span>	<span class="hljs-number">2070</span><br><span class="hljs-attribute">13956435636</span>	<span class="hljs-number">132</span>	<span class="hljs-number">1512</span>	<span class="hljs-number">1644</span><br><span class="hljs-attribute">13470253144</span>	<span class="hljs-number">180</span>	<span class="hljs-number">180</span>	<span class="hljs-number">360</span><br><span class="hljs-attribute">13846544121</span>	<span class="hljs-number">264</span>	<span class="hljs-number">0</span>	<span class="hljs-number">264</span><br><span class="hljs-attribute">13729199489</span>	<span class="hljs-number">240</span>	<span class="hljs-number">0</span>	<span class="hljs-number">240</span><br><span class="hljs-attribute">13966251146</span>	<span class="hljs-number">240</span>	<span class="hljs-number">0</span>	<span class="hljs-number">240</span><br><span class="hljs-attribute">13768778790</span>	<span class="hljs-number">120</span>	<span class="hljs-number">120</span>	<span class="hljs-number">240</span><br></code></pre></td></tr></table></figure>


<h1 id="五、Combiner-合并"><a href="#五、Combiner-合并" class="headerlink" title="五、Combiner 合并"></a>五、Combiner 合并</h1><hr>
<h2 id="5-1-基本概念"><a href="#5-1-基本概念" class="headerlink" title="5.1 基本概念"></a>5.1 基本概念</h2><blockquote>
<p>Combiner 是MR程序汇总Mapper和Reducer之外的一种组件</p>
</blockquote>
<blockquote>
<p>Combiner 组件的父类就是Reducer</p>
</blockquote>
<blockquote>
<p>Combiner和Reducer的区别在于运行的位置：</p>
<ul>
<li>Combiner是在每一个MapTask所在的节点运行</li>
<li>Reducer是接收全局所有Mapper的输出结果</li>
</ul>
</blockquote>
<blockquote>
<p>Combiner的意义就是对每一个MapTask的输出进行局部汇总，以减小网络传输量</p>
</blockquote>
<blockquote>
<p>Combiner能应用的前提是不能影响最终的业务逻辑，而且，Combiner的输出kv类型应该和Reducer的输入kv类型对应</p>
</blockquote>
<p>举个例：以下场景就无法使用Combiner！<br>在mapper阶段：</p>
<figure class="highlight basic"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs basic"><span class="hljs-symbol">3 </span><span class="hljs-number">5</span> <span class="hljs-number">7</span> -&gt; (<span class="hljs-number">3</span> + <span class="hljs-number">5</span> + <span class="hljs-number">7</span>）/ <span class="hljs-number">3</span> = <span class="hljs-number">5</span><br><span class="hljs-symbol">2 </span><span class="hljs-number">6</span> -&gt; (<span class="hljs-number">2</span> + <span class="hljs-number">6</span>) / <span class="hljs-number">2</span> = <span class="hljs-number">4</span><br></code></pre></td></tr></table></figure>
<p>在Reducer阶段：</p>
<figure class="highlight subunit"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs subunit">(3<span class="hljs-string">+5</span><span class="hljs-string">+7</span><span class="hljs-string">+2</span><span class="hljs-string">+6</span>) / 5 = 23 / 5<br></code></pre></td></tr></table></figure>
<p>其中，Reducer的结果和Combiner计算的(5+4) &#x2F; 2 !&#x3D; 9&#x2F;2 结果不相同</p>
<h2 id="5-2-Combiner-合并案例"><a href="#5-2-Combiner-合并案例" class="headerlink" title="5.2 Combiner 合并案例"></a>5.2 Combiner 合并案例</h2><p><strong>案例：</strong>  以经典的词频统计为例，统计过程中对每一个MapTask的输出进行局部汇总，以减小网络传输量即采用Combiner功能</p>
<p><strong>思路1：</strong></p>
<ul>
<li>增加一个WordcountCombiner类继承Reducer</li>
<li>在WordcountCombiner中统计单词汇总、将统计结果输出</li>
</ul>
<p><strong>思路2：</strong></p>
<ul>
<li>将WordcountReducer作为Combiner在WordcountDriver驱动类中设置<code>job.setCombinerClass(WordcountReducer.class)</code></li>
</ul>
<p>测试文本：</p>
<figure class="highlight mipsasm"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><code class="hljs mipsasm">hello world<br>hello hadoop<br><span class="hljs-keyword">java </span><span class="hljs-keyword">and </span>hadoop<br>hdfs <span class="hljs-keyword">and </span>mapreduce<br></code></pre></td></tr></table></figure>

<p><code>WordCountMapper.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.combiner;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.IntWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.LongWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Mapper;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">WordCountMapper</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">Mapper</span>&lt;LongWritable, Text, Text, IntWritable&gt; &#123;<br>    <span class="hljs-comment">// 放在上面声明防止在循环里多次创建对象，浪费空间</span><br>    <span class="hljs-keyword">private</span> <span class="hljs-type">Text</span> <span class="hljs-variable">outKey</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Text</span>();<br>    <span class="hljs-keyword">private</span> <span class="hljs-type">IntWritable</span> <span class="hljs-variable">outValue</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">IntWritable</span>(<span class="hljs-number">1</span>);<br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">protected</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">map</span><span class="hljs-params">(LongWritable key, Text value, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>        <span class="hljs-comment">// 1. 获取一行</span><br>        <span class="hljs-type">String</span> <span class="hljs-variable">line</span> <span class="hljs-operator">=</span> value.toString();<br>        <span class="hljs-comment">// 2. 切割</span><br>        String[] words = line.split(<span class="hljs-string">&quot; &quot;</span>);<br>        <span class="hljs-comment">// 3. 循环写出</span><br>        <span class="hljs-keyword">for</span> (String word : words) &#123;<br>            <span class="hljs-comment">// 封装 outKey</span><br>            outKey.set(word);<br>            context.write(outKey, outValue);<br>        &#125;<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>


<p><code>WordCountReducer.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.combiner;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.IntWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Reducer;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">WordCountReducer</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">Reducer</span>&lt;Text, IntWritable, Text, IntWritable&gt; &#123;<br>    <span class="hljs-keyword">private</span>  <span class="hljs-type">IntWritable</span> <span class="hljs-variable">outValue</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">IntWritable</span>();<br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">protected</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">reduce</span><span class="hljs-params">(Text key, Iterable&lt;IntWritable&gt; values, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>        <span class="hljs-type">int</span> <span class="hljs-variable">sum</span> <span class="hljs-operator">=</span> <span class="hljs-number">0</span>;<br>        <span class="hljs-comment">// 累加</span><br>        <span class="hljs-keyword">for</span> (IntWritable value : values) &#123;<br>            sum += value.get();<br>        &#125;<br>        outValue.set(sum);<br>        <span class="hljs-comment">// 写出</span><br>        context.write(key, outValue);<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>

<p><code>WordCountCombiner.java</code>（可用Reducer替换）</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.combiner;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.IntWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Reducer;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">WordCountCombiner</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">Reducer</span>&lt;Text, IntWritable, Text, IntWritable&gt; &#123;<br>    <span class="hljs-keyword">private</span> <span class="hljs-type">IntWritable</span> <span class="hljs-variable">outputValue</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">IntWritable</span>();<br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">protected</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">reduce</span><span class="hljs-params">(Text key, Iterable&lt;IntWritable&gt; values, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>        <span class="hljs-type">int</span> <span class="hljs-variable">sum</span> <span class="hljs-operator">=</span> <span class="hljs-number">0</span>;<br>        <span class="hljs-keyword">for</span> (IntWritable value : values) &#123;<br>            sum += value.get();<br>        &#125;<br>        outputValue.set(sum);<br>        context.write(key, outputValue);<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>

<p><code>WordCountDriver.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.combiner;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.conf.Configuration;<br><span class="hljs-keyword">import</span> org.apache.hadoop.fs.Path;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.IntWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Job;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.lib.input.FileInputFormat;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">WordCountDriver</span>&#123;<br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">static</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">main</span><span class="hljs-params">(String[] args)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException, ClassNotFoundException &#123;<br>        <span class="hljs-comment">// 1. 获取 job</span><br>        <span class="hljs-type">Configuration</span> <span class="hljs-variable">conf</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Configuration</span>();<br>        <span class="hljs-type">Job</span> <span class="hljs-variable">job</span> <span class="hljs-operator">=</span> Job.getInstance(conf);<br>        <span class="hljs-comment">// 2. 设置jar包路径</span><br>        job.setJarByClass(WordCountDriver.class);<br>        <span class="hljs-comment">// 3. 关联 mapper 和 reducer</span><br>        job.setMapperClass(WordCountMapper.class);<br>        job.setReducerClass(WordCountReducer.class);<br>        <span class="hljs-comment">// 4. 设置 map 输出的 k v 类型</span><br>        job.setMapOutputKeyClass(Text.class);<br>        job.setMapOutputValueClass(IntWritable.class);<br>        <span class="hljs-comment">// 5. 设置最终输出的k v类型</span><br>        job.setOutputKeyClass(Text.class);<br>        job.setOutputValueClass(IntWritable.class);<br>        <span class="hljs-comment">// 设置Combiner组件</span><br>        job.setCombinerClass(WordCountCombiner.class);<br>        <span class="hljs-comment">// 6. 设置输入路径和输出路径</span><br>        FileInputFormat.setInputPaths(job,<span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;input4&quot;</span>));<br>        FileOutputFormat.setOutputPath(job, <span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;output4&quot;</span>));<br>        <span class="hljs-comment">// 7. 提交 job</span><br>        <span class="hljs-type">boolean</span> <span class="hljs-variable">result</span> <span class="hljs-operator">=</span> job.waitForCompletion(<span class="hljs-literal">true</span>);<br>        System.exit(result ? <span class="hljs-number">0</span> : <span class="hljs-number">1</span>);<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>

<p>程序运行结果：</p>
<figure class="highlight apache"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><code class="hljs apache"><span class="hljs-attribute">and</span>	<span class="hljs-number">2</span><br><span class="hljs-attribute">hadoop</span>	<span class="hljs-number">2</span><br><span class="hljs-attribute">hdfs</span>	<span class="hljs-number">1</span><br><span class="hljs-attribute">hello</span>	<span class="hljs-number">2</span><br><span class="hljs-attribute">java</span>	<span class="hljs-number">1</span><br><span class="hljs-attribute">mapreduce</span>	<span class="hljs-number">1</span><br><span class="hljs-attribute">world</span>	<span class="hljs-number">1</span><br></code></pre></td></tr></table></figure>

<p>使用Combiner前后对比结果：</p>
<p><img src="https://img-blog.csdnimg.cn/a2f58014e09546af82005f38974affc5.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"><br>观察代码，可以发现在本次案例中，Combiner和Reducer的内容一致，都是Reducer类的子类，所以在Driver代码块里，可以将：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs java">job.setCombinerClass(WordCountCombiner.class);<br></code></pre></td></tr></table></figure>
<p>替换成：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs java">job.setCombinerClass(WordCountReducer.class);<br></code></pre></td></tr></table></figure>

<p>这就使得<code>WordCountReducer.java</code>一个类同时作为Combiner和Reducer的执行类，所以在设置combiner时，只要符合业务逻辑，通常设置为reducer类，无需再写一个Combiner类</p>
<h1 id="六、OutputFormat接口实现类"><a href="#六、OutputFormat接口实现类" class="headerlink" title="六、OutputFormat接口实现类"></a>六、OutputFormat接口实现类</h1><hr>
<h2 id="6-1-基本概念"><a href="#6-1-基本概念" class="headerlink" title="6.1 基本概念"></a>6.1 基本概念</h2><p>outputFormat是MapReduce输出的 <strong>基类</strong>，所有实现MapReduce输出 <strong>都</strong>实现了OutputFormat接口，下面是常见的几种OutputFormat实现类<br><img src="https://img-blog.csdnimg.cn/b691238f65f74bbeb7e7fb730973b45d.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<ol>
<li>使用OutputFormat实现类</li>
<li>使用默认输出格式TextOutputFormat</li>
<li>使用自定义OutputFormat（应用场景：例如输出数据到MySQL&#x2F;HBase&#x2F;Elasticsearch等存储框架中）</li>
</ol>
<p>自定义OutputFormat步骤：</p>
<ol>
<li>自定义一个类继承fileoutputformat</li>
<li>改写RecordWriter，具体改写输出数据的方法write()</li>
</ol>
<h2 id="6-2-自定义OutputFormat案例"><a href="#6-2-自定义OutputFormat案例" class="headerlink" title="6.2 自定义OutputFormat案例"></a>6.2 自定义OutputFormat案例</h2><p><strong>案例：</strong> 过滤测试的<code>test.log</code>日志，将包含uni内容的网站输出到本地的一个日志文件<code>uni.log</code>里，不包含uni的其他网站输出到另外的日志文件<code>nouni.log</code></p>
<p><strong>思路：</strong> 创建一个类继承RecordWriter，在类里实现：创建两个输出日志文件的输出流，根据输出流写入相应的内容</p>
<h3 id="6-2-1-编写RecordWriter继承类"><a href="#6-2-1-编写RecordWriter继承类" class="headerlink" title="6.2.1 编写RecordWriter继承类"></a>6.2.1 编写RecordWriter继承类</h3><p><code>LogRecordWriter.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.outputformat;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.fs.FSDataOutputStream;<br><span class="hljs-keyword">import</span> org.apache.hadoop.fs.FileSystem;<br><span class="hljs-keyword">import</span> org.apache.hadoop.fs.Path;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.IOUtils;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.NullWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.RecordWriter;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.TaskAttemptContext;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">LogRecordWriter</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">RecordWriter</span>&lt;Text, NullWritable&gt; &#123;<br><br>    <span class="hljs-keyword">private</span> FSDataOutputStream uniOut;<br>    <span class="hljs-keyword">private</span> FSDataOutputStream noOut;<br><br>    <span class="hljs-keyword">public</span> <span class="hljs-title function_">LogRecordWriter</span><span class="hljs-params">(TaskAttemptContext job)</span>&#123;<br>        <span class="hljs-comment">// 创建两个日志文件流</span><br>        <span class="hljs-keyword">try</span> &#123;<br>            <span class="hljs-type">FileSystem</span> <span class="hljs-variable">fs</span> <span class="hljs-operator">=</span> FileSystem.get(job.getConfiguration());<br>            uniOut = fs.create(<span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;uni.log&quot;</span>));<br>            noOut = fs.create(<span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;nouni.log&quot;</span>));<br><br>        &#125; <span class="hljs-keyword">catch</span> (IOException e) &#123;<br>            e.printStackTrace();<br>        &#125;<br>    &#125;<br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">write</span><span class="hljs-params">(Text key, NullWritable value)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>        <span class="hljs-type">String</span> <span class="hljs-variable">log</span> <span class="hljs-operator">=</span> key.toString();<br>        <span class="hljs-keyword">if</span>(log.contains(<span class="hljs-string">&quot;uni&quot;</span>))&#123;<br>            uniOut.writeBytes(log + <span class="hljs-string">&quot;\n&quot;</span>);<br>        &#125; <span class="hljs-keyword">else</span><br>            noOut.writeBytes(log + <span class="hljs-string">&quot;\n&quot;</span>);<br>    &#125;<br><br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">close</span><span class="hljs-params">(TaskAttemptContext context)</span>&#123;<br>        IOUtils.closeStream(uniOut);<br>        IOUtils.closeStream(noOut);<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>
<h3 id="6-2-2-编写FileOutputFormat继承类"><a href="#6-2-2-编写FileOutputFormat继承类" class="headerlink" title="6.2.2 编写FileOutputFormat继承类"></a>6.2.2 编写FileOutputFormat继承类</h3><p><code>LogOutputFormat.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.outputformat;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.fs.FileSystem;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.NullWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.RecordWriter;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.TaskAttemptContext;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;<br><span class="hljs-keyword">import</span> org.apache.hadoop.util.Progressable;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">LogOutputFormat</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">FileOutputFormat</span>&lt;Text, NullWritable&gt; &#123;<br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">public</span> RecordWriter&lt;Text, NullWritable&gt; <span class="hljs-title function_">getRecordWriter</span><span class="hljs-params">(TaskAttemptContext job)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>        <span class="hljs-type">LogRecordWriter</span> <span class="hljs-variable">lrw</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">LogRecordWriter</span>(job);<br>        <span class="hljs-keyword">return</span> lrw;<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>
<h3 id="6-2-3-Mapper-amp-Reducer-amp-Driver"><a href="#6-2-3-Mapper-amp-Reducer-amp-Driver" class="headerlink" title="6.2.3 Mapper &amp; Reducer &amp; Driver"></a>6.2.3 Mapper &amp; Reducer &amp; Driver</h3><p><code>LogMapper.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.outputformat;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.LongWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.NullWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Mapper;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">LogMapper</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">Mapper</span>&lt;LongWritable, Text, Text, NullWritable&gt; &#123;<br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">protected</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">map</span><span class="hljs-params">(LongWritable key, Text value, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>        <span class="hljs-comment">// Map阶段不做处理</span><br>        context.write(value, NullWritable.get());<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>
<p><code>LogReducer.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.outputformat;<br><br><span class="hljs-keyword">import</span> org.apache.hadoop.io.NullWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Reducer;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">LogReducer</span> <span class="hljs-keyword">extends</span> <span class="hljs-title class_">Reducer</span>&lt;Text, NullWritable, Text, NullWritable&gt; &#123;<br>    <span class="hljs-meta">@Override</span><br>    <span class="hljs-keyword">protected</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">reduce</span><span class="hljs-params">(Text key, Iterable&lt;NullWritable&gt; values, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException &#123;<br>        <span class="hljs-comment">// 防止有相同数据，丢数据</span><br>        <span class="hljs-keyword">for</span> (NullWritable value : values) &#123;<br>            context.write(key, NullWritable.get());<br>        &#125;<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>
<p><code>Driver.java</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">package</span> com.uni.outputformat;<br><br><span class="hljs-keyword">import</span> com.uni.writableComparable.FlowBean;<br><span class="hljs-keyword">import</span> com.uni.writableComparable.FlowDriver;<br><span class="hljs-keyword">import</span> com.uni.writableComparable.FlowMapper;<br><span class="hljs-keyword">import</span> com.uni.writableComparable.FlowReducer;<br><span class="hljs-keyword">import</span> org.apache.hadoop.conf.Configuration;<br><span class="hljs-keyword">import</span> org.apache.hadoop.fs.Path;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.NullWritable;<br><span class="hljs-keyword">import</span> org.apache.hadoop.io.Text;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.Job;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.lib.input.FileInputFormat;<br><span class="hljs-keyword">import</span> org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;<br><br><span class="hljs-keyword">import</span> java.io.IOException;<br><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">LogDriver</span> &#123;<br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">static</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">main</span><span class="hljs-params">(String[] args)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException, ClassNotFoundException &#123;<br>       <br>        <span class="hljs-type">Configuration</span> <span class="hljs-variable">conf</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Configuration</span>();<br>        <span class="hljs-type">Job</span> <span class="hljs-variable">job</span> <span class="hljs-operator">=</span> Job.getInstance(conf);<br>  <br>        job.setJarByClass(FlowDriver.class);<br>       <br>        job.setMapperClass(LogMapper.class);<br>        job.setReducerClass(LogReducer.class);<br>   <br>        job.setMapOutputKeyClass(Text.class);<br>        job.setMapOutputValueClass(NullWritable.class);<br>       <br>        job.setOutputKeyClass(Text.class);<br>        job.setOutputValueClass(NullWritable.class);<br><br>        <span class="hljs-comment">// 设置自定义的outputFormat</span><br>        job.setOutputFormatClass(LogOutputFormat.class);<br>        <span class="hljs-comment">// 设置数据的输入路径和输出路径</span><br>        FileInputFormat.setInputPaths(job, <span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;input5&quot;</span>));<br>        FileOutputFormat.setOutputPath(job, <span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;output5&quot;</span>));<br>       <br>        <span class="hljs-type">boolean</span> <span class="hljs-variable">result</span> <span class="hljs-operator">=</span> job.waitForCompletion(<span class="hljs-literal">true</span>);<br>        System.exit(result ? <span class="hljs-number">0</span> : <span class="hljs-number">1</span>);<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>
<p>运行结果：<br><code>uni.log</code></p>
<figure class="highlight awk"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs awk">https:<span class="hljs-regexp">//</span>www.uni.cn<br>https:<span class="hljs-regexp">//</span>www.unitest.cn<br></code></pre></td></tr></table></figure>
<p><code>nouni.log</code></p>
<figure class="highlight awk"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><code class="hljs awk">https:<span class="hljs-regexp">//</span>localhost:<span class="hljs-number">80</span><br>https:<span class="hljs-regexp">//</span>localhost:<span class="hljs-number">8080</span><br>https:<span class="hljs-regexp">//</span>www.baidu.com<br>https:<span class="hljs-regexp">//</span>www.csdn.net<br></code></pre></td></tr></table></figure>

                
              </div>
            
            <hr/>
            <div>
              <div class="post-metas my-3">
  
    <div class="post-meta mr-3 d-flex align-items-center">
      <i class="iconfont icon-category"></i>
      

<span class="category-chains">
  
  
    
      <span class="category-chain">
        
  <a href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/" class="category-chain-item">大数据</a>
  
  

      </span>
    
  
</span>

    </div>
  
  
    <div class="post-meta">
      <i class="iconfont icon-tags"></i>
      
        <a href="/tags/MapReduce/">#MapReduce</a>
      
    </div>
  
</div>


              
  

  <div class="license-box my-3">
    <div class="license-title">
      <div>Hadoop _ MapReduce学习笔记 _ Partitioner分区 自定义分区策略案例 _ WritableComparable 全排序 _ Combiner、OutputFormat</div>
      <div>http://example.com/2022/02/13/Hadoop _ MapReduce学习笔记 _ Partitioner分区 自定义分区策略案例 _ WritableComparable 全排序 _ Combiner、OutputFormat/</div>
    </div>
    <div class="license-meta">
      
        <div class="license-meta-item">
          <div>作者</div>
          <div>John Doe</div>
        </div>
      
      
        <div class="license-meta-item license-meta-date">
          <div>发布于</div>
          <div>2022年2月13日</div>
        </div>
      
      
      <div class="license-meta-item">
        <div>许可协议</div>
        <div>
          
            
            
              <a target="_blank" href="https://creativecommons.org/licenses/by/4.0/">
              <span class="hint--top hint--rounded" aria-label="BY - 署名">
                <i class="iconfont icon-by"></i>
              </span>
              </a>
            
          
        </div>
      </div>
    </div>
    <div class="license-icon iconfont"></div>
  </div>



              
                <div class="post-prevnext my-3">
                  <article class="post-prev col-6">
                    
                    
                      <a href="/2022/02/17/Web%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98%20_%20%E8%B4%AD%E7%89%A9%E7%B3%BB%E7%BB%9Fv2.0%20_%20%E5%BC%80%E5%8F%91%E8%AE%B0%E5%BD%95%EF%BC%88%E5%85%AB%EF%BC%89%E5%89%8D%E5%90%8E%E7%AB%AF%E5%88%86%E7%A6%BB%E5%88%9D%E6%AD%A5%E6%80%9D%E8%B7%AF%20_%20JS%E5%A4%84%E7%90%86URL%E5%8F%82%E6%95%B0%E5%AE%9E%E7%94%A8%E5%87%BD%E6%95%B0%20_%20AJAX%20%E5%90%91%E5%90%8E%E5%8F%B0%E4%BC%A0%E9%80%92Map%E7%B1%BB%E5%9E%8B%E6%95%B0%E6%8D%AE%20_%20MyBatis%E5%A4%9A%E8%A1%A8%E6%9F%A5%E8%AF%A2%E4%BC%98%E5%8C%96/" title="Web项目实战 _ 购物系统v2.0 _ 开发记录（八）前后端分离初步思路 _ JS处理URL参数实用函数 _ AJAX 向后台传递Map类型数据 _ MyBatis多表查询优化">
                        <i class="iconfont icon-arrowleft"></i>
                        <span class="hidden-mobile">Web项目实战 _ 购物系统v2.0 _ 开发记录（八）前后端分离初步思路 _ JS处理URL参数实用函数 _ AJAX 向后台传递Map类型数据 _ MyBatis多表查询优化</span>
                        <span class="visible-mobile">上一篇</span>
                      </a>
                    
                  </article>
                  <article class="post-next col-6">
                    
                    
                      <a href="/2022/02/12/Web%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98%20_%20%E8%B4%AD%E7%89%A9%E7%B3%BB%E7%BB%9Fv2.0%20_%20%E5%BC%80%E5%8F%91%E8%AE%B0%E5%BD%95%EF%BC%88%E4%B8%83%EF%BC%89SpringBoot%E6%95%B4%E5%90%88Shiro%E6%A1%86%E6%9E%B6%E8%BF%9B%E8%A1%8C%E8%BA%AB%E4%BB%BD%E8%AE%A4%E8%AF%81%20_%20Shiro%20%E5%8A%A0%E7%9B%90(MD5+Salt)%E9%AA%8C%E8%AF%81%E7%99%BB%E9%99%86%20_%20%E6%95%B0%E6%8D%AE%E8%A1%A8%E7%BB%93%E6%9E%84%E4%BC%98%E5%8C%96%E9%81%BF%E5%85%8D%E5%A4%96%E9%94%AE+%E8%AE%BE%E7%BD%AE%E4%B8%AD%E9%97%B4%E8%A1%A8/" title="Web项目实战 _ 购物系统v2.0 _ 开发记录（七）SpringBoot整合Shiro框架进行身份认证 _ Shiro 加盐(MD5+Salt)验证登陆 _ 数据表结构优化避免外键+设置中间表">
                        <span class="hidden-mobile">Web项目实战 _ 购物系统v2.0 _ 开发记录（七）SpringBoot整合Shiro框架进行身份认证 _ Shiro 加盐(MD5+Salt)验证登陆 _ 数据表结构优化避免外键+设置中间表</span>
                        <span class="visible-mobile">下一篇</span>
                        <i class="iconfont icon-arrowright"></i>
                      </a>
                    
                  </article>
                </div>
              
            </div>

            
          </article>
        </div>
      </div>
    </div>

    <div class="side-col d-none d-lg-block col-lg-2">
      
  <aside class="sidebar" style="margin-left: -1rem">
    <div id="toc">
  <p class="toc-header"><i class="iconfont icon-list"></i>&nbsp;目录</p>
  <div class="toc-body" id="toc-body"></div>
</div>



  </aside>


    </div>
  </div>
</div>





  



  



  



  



  







    

    
      <a id="scroll-top-button" aria-label="TOP" href="#" role="button">
        <i class="iconfont icon-arrowup" aria-hidden="true"></i>
      </a>
    

    
      <div class="modal fade" id="modalSearch" tabindex="-1" role="dialog" aria-labelledby="ModalLabel"
     aria-hidden="true">
  <div class="modal-dialog modal-dialog-scrollable modal-lg" role="document">
    <div class="modal-content">
      <div class="modal-header text-center">
        <h4 class="modal-title w-100 font-weight-bold">搜索</h4>
        <button type="button" id="local-search-close" class="close" data-dismiss="modal" aria-label="Close">
          <span aria-hidden="true">&times;</span>
        </button>
      </div>
      <div class="modal-body mx-3">
        <div class="md-form mb-5">
          <input type="text" id="local-search-input" class="form-control validate">
          <label data-error="x" data-success="v" for="local-search-input">关键词</label>
        </div>
        <div class="list-group" id="local-search-result"></div>
      </div>
    </div>
  </div>
</div>

    

    
  </main>

  <footer>
    <div class="footer-inner">
  
    <div class="footer-content">
       <a href="https://hexo.io" target="_blank" rel="nofollow noopener"><span>Hexo</span></a> <i class="iconfont icon-love"></i> <a href="https://github.com/fluid-dev/hexo-theme-fluid" target="_blank" rel="nofollow noopener"><span>Fluid</span></a> 
    </div>
  
  
  
  
</div>

  </footer>

  <!-- Scripts -->
  
  <script  src="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.js" ></script>
  <link  rel="stylesheet" href="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.css" />

  <script>
    NProgress.configure({"showSpinner":false,"trickleSpeed":100})
    NProgress.start()
    window.addEventListener('load', function() {
      NProgress.done();
    })
  </script>


<script  src="https://lib.baomitu.com/jquery/3.6.0/jquery.min.js" ></script>
<script  src="https://lib.baomitu.com/twitter-bootstrap/4.6.1/js/bootstrap.min.js" ></script>
<script  src="/js/events.js" ></script>
<script  src="/js/plugins.js" ></script>


  <script  src="https://lib.baomitu.com/typed.js/2.0.12/typed.min.js" ></script>
  <script>
    (function (window, document) {
      var typing = Fluid.plugins.typing;
      var subtitle = document.getElementById('subtitle');
      if (!subtitle || !typing) {
        return;
      }
      var text = subtitle.getAttribute('data-typed-text');
      
        typing(text);
      
    })(window, document);
  </script>




  
    <script  src="/js/img-lazyload.js" ></script>
  




  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/tocbot/4.18.2/tocbot.min.js', function() {
    var toc = jQuery('#toc');
    if (toc.length === 0 || !window.tocbot) { return; }
    var boardCtn = jQuery('#board-ctn');
    var boardTop = boardCtn.offset().top;

    window.tocbot.init({
      tocSelector     : '#toc-body',
      contentSelector : '.markdown-body',
      headingSelector : CONFIG.toc.headingSelector || 'h1,h2,h3,h4,h5,h6',
      linkClass       : 'tocbot-link',
      activeLinkClass : 'tocbot-active-link',
      listClass       : 'tocbot-list',
      isCollapsedClass: 'tocbot-is-collapsed',
      collapsibleClass: 'tocbot-is-collapsible',
      collapseDepth   : CONFIG.toc.collapseDepth || 0,
      scrollSmooth    : true,
      headingsOffset  : -boardTop
    });
    if (toc.find('.toc-list-item').length > 0) {
      toc.css('visibility', 'visible');
    }
  });
</script>


  <script src=https://lib.baomitu.com/clipboard.js/2.0.10/clipboard.min.js></script>

  <script>Fluid.plugins.codeWidget();</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/anchor-js/4.3.1/anchor.min.js', function() {
    window.anchors.options = {
      placement: CONFIG.anchorjs.placement,
      visible  : CONFIG.anchorjs.visible
    };
    if (CONFIG.anchorjs.icon) {
      window.anchors.options.icon = CONFIG.anchorjs.icon;
    }
    var el = (CONFIG.anchorjs.element || 'h1,h2,h3,h4,h5,h6').split(',');
    var res = [];
    for (var item of el) {
      res.push('.markdown-body > ' + item.trim());
    }
    if (CONFIG.anchorjs.placement === 'left') {
      window.anchors.options.class = 'anchorjs-link-left';
    }
    window.anchors.add(res.join(', '));
  });
</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.js', function() {
    Fluid.plugins.fancyBox();
  });
</script>


  <script>Fluid.plugins.imageCaption();</script>

  <script  src="/js/local-search.js" ></script>





<!-- 主题的启动项，将它保持在最底部 -->
<!-- the boot of the theme, keep it at the bottom -->
<script  src="/js/boot.js" ></script>


  

  <noscript>
    <div class="noscript-warning">博客在允许 JavaScript 运行的环境下浏览效果更佳</div>
  </noscript>
</body>
</html>
